package bcnet.service;

import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Properties;
import java.util.TreeMap;
import java.util.logging.Level;

import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.AddressException;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;

import bcnet.dao.BoughtCodeDao;
import bcnet.dao.CodeDao;
import bcnet.dao.PriceHistoryDao;
import bcnet.domain.BoughtCodeVO;
import bcnet.domain.CodeVO;
import bcnet.domain.PriceHistoryVO;
import bcnet.util.RSquare;

import com.google.appengine.api.memcache.ErrorHandlers;
import com.google.appengine.api.memcache.Expiration;
import com.google.appengine.api.memcache.MemcacheService;
import com.google.appengine.api.memcache.MemcacheServiceFactory;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;


public class PriceHistoryService {
	static PriceHistoryDao priceHistoryDao;
	static CodeDao codeDao;
	static MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService();
    
	
	public void addPriceHistory(String code, Date priceDate, double closePrice, long volume, double lowPrice, double highPrice){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		
		PriceHistoryVO priceHistoryVO = new PriceHistoryVO();
		priceHistoryVO.setClosePrice(closePrice);
		priceHistoryVO.setCode(code);
		priceHistoryVO.setPriceDate(priceDate);
		priceHistoryVO.setVolume(volume);
		priceHistoryVO.setLowPrice(lowPrice);
		priceHistoryVO.setHighPrice(highPrice);
		
		priceHistoryDao = new PriceHistoryDao();
		
		priceHistoryDao.addPriceHistory(priceHistoryVO);
		
		// remove the codeListString cache to let hasHistory to reload the list
		syncCache.delete("codeListString");
	}
	

	// for real time get chart string
	public String getPriceChartString(String code, Double np){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    String cachedChartString = (String) syncCache.get("chartString_"+code);
	    int cachedAvgCount = (Integer) syncCache.get("avgCount_"+code)==null?0:(Integer) syncCache.get("avgCount_"+code);
	    double precision = (Double) syncCache.get("precision_"+code)==null?0.0:(Double) syncCache.get("precision_"+code);
	    double cahcedMiddlePrice = (Double) syncCache.get("middlePrice_"+code)==null?0.0:(Double) syncCache.get("middlePrice_"+code);
	    DecimalFormat f = new DecimalFormat("##.#");
	    String tempString;
		// no cache, get from yahoo
		if (cachedChartString == null || cachedAvgCount == 0 || precision == 0.0 || cahcedMiddlePrice == 0.0){
			Calendar cal = Calendar.getInstance();
			//if (SystemProperty.environment.value() == SystemProperty.Environment.Value.Production) {
			cal.add(Calendar.HOUR, +8);
			//}
			Date todayDate = cal.getTime();
			SimpleDateFormat sdfYear = new SimpleDateFormat("yyyy");
			SimpleDateFormat sdfMonth = new SimpleDateFormat("MM");
			SimpleDateFormat sdfDate = new SimpleDateFormat("dd");
			
			int toMonthInt = Integer.parseInt(sdfMonth.format(todayDate));
			toMonthInt--;
			
			String toYearString = sdfYear.format(todayDate);
			String toMonthString = String.format("%02d", toMonthInt);
			String toDateString = sdfDate.format(todayDate);
			
			cal.add(Calendar.YEAR, -1);
			Date lastYearDate = cal.getTime();
			
			int fromMonthInt = Integer.parseInt(sdfMonth.format(lastYearDate));
			fromMonthInt--;

			String fromYearString = sdfYear.format(lastYearDate);
			String fromMonthString = String.format("%02d", fromMonthInt);
			String fromDateString = sdfDate.format(lastYearDate); 

			
			List <PriceHistoryVO> priceHistoryList = getPriceHistoryVOList(code, fromYearString,fromMonthString,fromDateString,toYearString,toMonthString,toDateString, false);
			tempString = getPriceChartString(priceHistoryList,np);
			return tempString;
		}
		else{ // have cache
			if(np <= 1.0){
				f = new DecimalFormat("###.##");
			}
			else if(np <= 10.0){
				f = new DecimalFormat("###.##");
			}
			else if (np <= 200.0){
				f = new DecimalFormat("###.#");
			}
			else if (np <= 2000.0){
				f = new DecimalFormat("###");
			}
			tempString = cachedChartString;
			tempString = tempString.replaceAll("\\["+f.format(np)+",null", "\\["+f.format(np)+","+cachedAvgCount);
			
			return tempString;
		}
	}
	
	// for cron to refresh chart string
	public String getPriceChartString(List<PriceHistoryVO> priceHistoryList, Double np){
		String code = priceHistoryList.get(0).getCode();
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		
	    String cachedChartString = (String) syncCache.get("chartString_"+code);
	    int cachedAvgCount = (Integer) syncCache.get("avgCount_"+code)==null?0:(Integer) syncCache.get("avgCount_"+code);
	    double precision = (Double) syncCache.get("precision_"+code)==null?0.0:(Double) syncCache.get("precision_"+code);
	    double cahcedMiddlePrice = (Double) syncCache.get("middlePrice_"+code)==null?0.0:(Double) syncCache.get("middlePrice_"+code);

	    DecimalFormat f = new DecimalFormat("##.#");
	    
	    String tempString;

		String formattedPrice = "";
		Double formattedPriceDouble = 0.0;
		Integer counter = 0;
		
		// no cache, get from DB
		if (cachedChartString == null || cachedAvgCount == 0 || precision == 0.0 || cahcedMiddlePrice == 0.0){
			//set the precision for better graph showing
			if(np <= 1.0){
				// Since yahoo price only minimum 2 dp, so no need cater for 3 dp even < $1
				precision = 0.01;
				f = new DecimalFormat("###.##");
			}
			else if(np <= 10.0){
				precision = 0.01;
				f = new DecimalFormat("###.##");
			}
			else if (np <= 200.0){
				precision = 0.1;
				f = new DecimalFormat("###.#");
			}
			else if (np <= 2000.0){
				precision = 1;
				f = new DecimalFormat("###");
			}
			
			Map<Double, Integer> priceHistoryMap = new HashMap<Double, Integer>();
			
			if(priceHistoryList.isEmpty()){
				return "";
			}
			else{
				for(int i=0; i<priceHistoryList.size(); i++){
					formattedPrice = f.format(priceHistoryList.get(i).getClosePrice());
					formattedPriceDouble = Double.parseDouble(formattedPrice);
					
					if(priceHistoryMap.containsKey(formattedPriceDouble)){
						counter = priceHistoryMap.get(formattedPriceDouble) + 1;
						priceHistoryMap.remove(formattedPriceDouble);
						priceHistoryMap.put(formattedPriceDouble, counter);
					}
					else {
						priceHistoryMap.put(formattedPriceDouble, 1);
					}
				}
				
				
				NavigableMap<Double, Integer> sortedPriceHistoryMap = new TreeMap<Double, Integer>(priceHistoryMap);
				StringBuilder sb = new StringBuilder();
				
				int avgCount = (Collections.max(sortedPriceHistoryMap.values()) + Collections.min(sortedPriceHistoryMap.values()))/2;
				double middlePrice = (sortedPriceHistoryMap.lastKey() + sortedPriceHistoryMap.firstKey())/2;
				
				for (double j=sortedPriceHistoryMap.firstKey(); j<=sortedPriceHistoryMap.lastKey(); j=(double)Math.round((j+precision)*100)/100) {
					sb.append("["+f.format(j)+",null,"+(sortedPriceHistoryMap.get(j)==null?"0":sortedPriceHistoryMap.get(j))+"],");
				}
				tempString = sb.toString();
				tempString = tempString.substring(0, tempString.length()-1);
				
				syncCache.put("chartString_"+code, tempString, Expiration.byDeltaSeconds(86400));
				syncCache.put("avgCount_"+code, avgCount,Expiration.byDeltaSeconds(86400));
				syncCache.put("precision_"+code, precision,Expiration.byDeltaSeconds(86400));
				syncCache.put("middlePrice_"+code, middlePrice,Expiration.byDeltaSeconds(86400));
				
				tempString = tempString.replaceAll("\\["+f.format(np)+",null", "\\["+f.format(np)+","+avgCount);
				//System.out.println("2No cache:" +tempString);
				
				return tempString;
			}
		}
		else{ // have cache
			if(np <= 1.0){
				f = new DecimalFormat("###.##");
			}
			else if(np <= 10.0){
				f = new DecimalFormat("###.##");
			}
			else if (np <= 200.0){
				f = new DecimalFormat("###.#");
			}
			else if (np <= 2000.0){
				f = new DecimalFormat("###");
			}
			tempString = cachedChartString;
			tempString = tempString.replaceAll("\\["+f.format(np)+",null", "\\["+f.format(np)+","+cachedAvgCount);
			
			return tempString;
		}
	}
	
	/*
	public String getVolumeChartString(String code, int day){
	    
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
	
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		
	    String cachedVolumeChartString = (String) syncCache.get("volumeChartString_"+code);
	    
	    String tempString;

	    List<PriceHistoryVO> priceHistoryList = null;

		// no cache, get from DB
		if (cachedVolumeChartString == null){
			
			priceHistoryDao = new PriceHistoryDao();
			priceHistoryList = priceHistoryDao.getPriceHistoryList(code,day);
						
			if(priceHistoryList.isEmpty()){
				return "";
			}
			else{
				StringBuilder sb = new StringBuilder();
				
				for(int i=priceHistoryList.size()-1; i>=0; i--){
					sb.append("['"+sdf.format(priceHistoryList.get(i).getPriceDate())+"',"+priceHistoryList.get(i).getVolume()+"],");
				}
				tempString = sb.toString();
				tempString = tempString.substring(0, tempString.length()-1);
				
				syncCache.put("volumeChartString_"+code, tempString);

				return tempString;
			}
		}
		else{ // have cache
			return cachedVolumeChartString;
		}
	}	
	 */
	
	public String getVolumeChartString(String code, long cv){
		long projectedVol = getProjectedVol(cv);
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    String cachedVolumeChartString2 = (String) syncCache.get("volumeChartString2_"+code);
	    int cachedVolumeAvgCount2 = (Integer) syncCache.get("volumeAvgCount2_"+code)==null?0:(Integer) syncCache.get("volumeAvgCount2_"+code);
	    int precision = (Integer) syncCache.get("volumePrecision2_"+code)==null?0:(Integer) syncCache.get("volumePrecision2_"+code);
	    double averageVolume = (Double) syncCache.get("volumeAverage_"+code)==null?0.0:(Double) syncCache.get("volumeAverage_"+code);
		double stdDevVolume = (Double) syncCache.get("volumeStdDev_"+code)==null?0.0:(Double) syncCache.get("volumeStdDev_"+code);
	    
	    String tempString;
	    String formattedVol = "";
		
		// no cache, get from DB
		if (cachedVolumeChartString2 == null || cachedVolumeAvgCount2 == 0 || precision == 0 || averageVolume == 0.0 || stdDevVolume == 0.0){
			//set the precision for better graph showing
			Calendar cal = Calendar.getInstance();
			//if (SystemProperty.environment.value() == SystemProperty.Environment.Value.Production) {
			cal.add(Calendar.HOUR, +8);
			//}
			Date todayDate = cal.getTime();
			SimpleDateFormat sdfYear = new SimpleDateFormat("yyyy");
			SimpleDateFormat sdfMonth = new SimpleDateFormat("MM");
			SimpleDateFormat sdfDate = new SimpleDateFormat("dd");
			int toMonthInt = Integer.parseInt(sdfMonth.format(todayDate));
			toMonthInt--;
			String toYearString = sdfYear.format(todayDate);
			String toMonthString = String.format("%02d", toMonthInt);
			String toDateString = sdfDate.format(todayDate);

			cal.add(Calendar.YEAR, -1);
			Date lastYearDate = cal.getTime();
			
			int fromMonthInt = Integer.parseInt(sdfMonth.format(lastYearDate));
			fromMonthInt--;

			String fromYearString = sdfYear.format(lastYearDate);
			String fromMonthString = String.format("%02d", fromMonthInt);
			String fromDateString = sdfDate.format(lastYearDate); 
			
			
			List <PriceHistoryVO> priceHistoryList = getPriceHistoryVOList(code, fromYearString,fromMonthString,fromDateString,toYearString,toMonthString,toDateString,false);
			tempString = getVolumeChartString(priceHistoryList,cv);
			//System.out.println("No Cache 1");
			return tempString;
		}
		else{ // have cache
			formattedVol = String.valueOf(Math.round(projectedVol/precision));
			tempString = cachedVolumeChartString2;
			tempString = tempString.replaceAll("\\["+formattedVol+",null", "\\["+formattedVol+","+cachedVolumeAvgCount2);
			//System.out.println("Have Cache 1");
			return tempString;
		}
	}
	
	public String getVolumeChartString(List<PriceHistoryVO> priceHistoryList, long cv){
		String code = priceHistoryList.get(0).getCode();
		long projectedVol = getProjectedVol(cv);
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    String cachedVolumeChartString2 = (String) syncCache.get("volumeChartString2_"+code);
	    int cachedVolumeAvgCount2 = (Integer) syncCache.get("volumeAvgCount2_"+code)==null?0:(Integer) syncCache.get("volumeAvgCount2_"+code);
	    int precision = (Integer) syncCache.get("volumePrecision2_"+code)==null?0:(Integer) syncCache.get("volumePrecision2_"+code);
	    double averageVolume = (Double) syncCache.get("volumeAverage_"+code)==null?0.0:(Double) syncCache.get("volumeAverage_"+code);
		double stdDevVolume = (Double) syncCache.get("volumeStdDev_"+code)==null?0.0:(Double) syncCache.get("volumeStdDev_"+code);
	    
	    String tempString;
	    String formattedVol = "";
	    Integer formattedVolume = 0;
		Integer counter = 0;
		
		// no cache, get from DB
		if (cachedVolumeChartString2 == null || cachedVolumeAvgCount2 == 0 || precision == 0 || averageVolume == 0.0 || stdDevVolume == 0.0){
			//set the precision for better graph showing
						
			Map<Integer, Integer> priceHistoryMap = new HashMap<Integer, Integer>();
			
			double volumeArray[] = new double[priceHistoryList.size()] ;
			
			if(priceHistoryList.isEmpty()){
				return "";
			}
			else{
				precision = 10000000;
				if(priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 100000000){
					precision = 1000000;
				}
				if (priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 10000000){
					precision = 100000;
				}
				if (priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 1000000){
					precision = 100000;
				}
				if (priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 100000){
					precision = 10000;
				}
				if (priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 10000){
					precision = 1000;
				}
				if (priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 1000){
					precision = 100;
				}
				if (priceHistoryList.get(priceHistoryList.size()-1).getVolume() <= 100){
					precision = 10;
				}			
				
				
				for(int i=0; i<priceHistoryList.size(); i++){
					
					formattedVolume = Math.round(priceHistoryList.get(i).getVolume()/precision);
					
					volumeArray[i] = priceHistoryList.get(i).getVolume();
					
					if(priceHistoryMap.containsKey(formattedVolume)){
						counter = priceHistoryMap.get(formattedVolume) + 1;
						//counter++;
						priceHistoryMap.remove(formattedVolume);
						priceHistoryMap.put(formattedVolume, counter);
					}
					else {
						priceHistoryMap.put(formattedVolume, 1);
					}
				}
				
				formattedVol = String.valueOf(Math.round(projectedVol/precision));
				
				averageVolume = getAverage(volumeArray);
				stdDevVolume = getStdDev(volumeArray);				
				
				NavigableMap<Integer, Integer> sortedPriceHistoryMap = new TreeMap<Integer, Integer>(priceHistoryMap);
				StringBuilder sb = new StringBuilder();
				
				
				int avgCount = (Collections.max(sortedPriceHistoryMap.values()) + Collections.min(sortedPriceHistoryMap.values()))/2;
				for (int j=sortedPriceHistoryMap.firstKey(); j<=sortedPriceHistoryMap.lastKey(); j++) {
					sb.append("["+j+",null,"+(sortedPriceHistoryMap.get(j)==null?"0":sortedPriceHistoryMap.get(j))+"],");
				}
				tempString = sb.toString();
				tempString = tempString.substring(0, tempString.length()-1);
				
				syncCache.put("volumeChartString2_"+code, tempString,Expiration.byDeltaSeconds(86400));
				syncCache.put("volumeAvgCount2_"+code, avgCount,Expiration.byDeltaSeconds(86400));
				syncCache.put("volumePrecision2_"+code, precision,Expiration.byDeltaSeconds(86400));
				
				syncCache.put("volumeAverage_"+code, averageVolume,Expiration.byDeltaSeconds(86400));
				syncCache.put("volumeStdDev_"+code, stdDevVolume,Expiration.byDeltaSeconds(86400));				
				
				tempString = tempString.replaceAll("\\["+formattedVol+",null", "\\["+formattedVol+","+avgCount);
				//System.out.println("No Cache 2");
				return tempString;
			}
		}
		else{ // have cache
			formattedVol = String.valueOf(Math.round(projectedVol/precision));
			tempString = cachedVolumeChartString2;
			tempString = tempString.replaceAll("\\["+formattedVol+",null", "\\["+formattedVol+","+cachedVolumeAvgCount2);
			//System.out.println("Have Cache 2");
			return tempString;
			
		}
	}
	
	
	public String getSpreadChartString(String code, Double np, Double spread){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    String cachedChartString = (String) syncCache.get("spreadChartString_"+code);
	    int cachedAvgCount = (Integer) syncCache.get("spreadAvgCount_"+code)==null?0:(Integer) syncCache.get("spreadAvgCount_"+code);
	    double precision = (Double) syncCache.get("spreadPrecision_"+code)==null?0.0:(Double) syncCache.get("spreadPrecision_"+code);
	    double averageSpread = (Double) syncCache.get("spreadAverage_"+code)==null?0.0:(Double) syncCache.get("spreadAverage_"+code);
		double stdDevSpread = (Double) syncCache.get("spreadStdDev_"+code)==null?0.0:(Double) syncCache.get("spreadStdDev_"+code);
		
	    DecimalFormat f = new DecimalFormat("##.#");
	    
	    String tempString;

		// no cache, get from DB
		if (cachedChartString == null || cachedAvgCount == 0 || precision == 0.0 || averageSpread == 0.0 || stdDevSpread == 0.0){
			Calendar cal = Calendar.getInstance();
			//if (SystemProperty.environment.value() == SystemProperty.Environment.Value.Production) {
			cal.add(Calendar.HOUR, +8);
			//}
			Date todayDate = cal.getTime();
			SimpleDateFormat sdfYear = new SimpleDateFormat("yyyy");
			SimpleDateFormat sdfMonth = new SimpleDateFormat("MM");
			SimpleDateFormat sdfDate = new SimpleDateFormat("dd");
			int toMonthInt = Integer.parseInt(sdfMonth.format(todayDate));
			toMonthInt--;
			String toYearString = sdfYear.format(todayDate);
			String toMonthString = String.format("%02d", toMonthInt);
			String toDateString = sdfDate.format(todayDate);
			
			cal.add(Calendar.YEAR, -1);
			Date lastYearDate = cal.getTime();
			
			int fromMonthInt = Integer.parseInt(sdfMonth.format(lastYearDate));
			fromMonthInt--;

			String fromYearString = sdfYear.format(lastYearDate);
			String fromMonthString = String.format("%02d", fromMonthInt);
			String fromDateString = sdfDate.format(lastYearDate); 
						
			
			List <PriceHistoryVO> priceHistoryList = getPriceHistoryVOList(code, fromYearString,fromMonthString,fromDateString,toYearString,toMonthString,toDateString,false);
			tempString = getSpreadChartString(priceHistoryList,np,spread);
			return tempString;
			
		}
		else{ // have cache
			if(np <= 1.0){
				f = new DecimalFormat("###.##");
			}
			else if(np <= 10.0){
				f = new DecimalFormat("###.##");
			}
			else if (np <= 200.0){
				f = new DecimalFormat("###.#");
			}
			else if (np <= 2000.0){
				f = new DecimalFormat("###");
			}
			tempString = cachedChartString;
			tempString = tempString.replaceAll("\\["+f.format(spread)+",null", "\\["+f.format(spread)+","+cachedAvgCount);
			return tempString;
		}
	}	
	
	public String getSpreadChartString(List<PriceHistoryVO> priceHistoryList, Double np, Double spread){
	 	String code = priceHistoryList.get(0).getCode();
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		
	    String cachedChartString = (String) syncCache.get("spreadChartString_"+code);
	    int cachedAvgCount = (Integer) syncCache.get("spreadAvgCount_"+code)==null?0:(Integer) syncCache.get("spreadAvgCount_"+code);
	    double precision = (Double) syncCache.get("spreadPrecision_"+code)==null?0.0:(Double) syncCache.get("spreadPrecision_"+code);
	    double averageSpread = (Double) syncCache.get("spreadAverage_"+code)==null?0.0:(Double) syncCache.get("spreadAverage_"+code);
		double stdDevSpread = (Double) syncCache.get("spreadStdDev_"+code)==null?0.0:(Double) syncCache.get("spreadStdDev_"+code);
		
	    DecimalFormat f = new DecimalFormat("##.#");
	    
	    String tempString;

		String formattedSpread = "";
		Double formattedSpreadDouble = 0.0;
		Integer counter = 0;
		// no cache, get from DB
		if (cachedChartString == null || cachedAvgCount == 0 || precision == 0.0 || averageSpread == 0.0 || stdDevSpread == 0.0){
			//set the precision for better graph showing
			if(np <= 1.0){
				// Since yahoo price only minimum 2 dp, so no need cater for 3 dp even < $1
				precision = 0.01;
				f = new DecimalFormat("###.##");
			}			
			else if(np <= 10.0){
				precision = 0.01;
				f = new DecimalFormat("###.##");
			}
			else if (np <= 200.0){
				precision = 0.1;
				f = new DecimalFormat("###.#");
			}
			else if (np <= 2000.0){
				precision = 1;
				f = new DecimalFormat("###");
			}
			
			Map<Double, Integer> priceHistoryMap = new HashMap<Double, Integer>();
			
			double spreadArray[] = new double[priceHistoryList.size()] ;
			
			if(priceHistoryList.isEmpty()){
				return "";
			}
			else{
				for(int i=0; i<priceHistoryList.size(); i++){
					formattedSpread = f.format(priceHistoryList.get(i).getHighPrice() - priceHistoryList.get(i).getLowPrice());
					
					spreadArray[i] = priceHistoryList.get(i).getHighPrice() - priceHistoryList.get(i).getLowPrice();
							
					formattedSpreadDouble = Double.parseDouble(formattedSpread);
					
					if(priceHistoryMap.containsKey(formattedSpreadDouble)){
						counter = priceHistoryMap.get(formattedSpreadDouble) + 1;
						//counter++;
						priceHistoryMap.remove(formattedSpreadDouble);
						priceHistoryMap.put(formattedSpreadDouble, counter);
					}
					else {
						priceHistoryMap.put(formattedSpreadDouble, 1);
					}
				}
				
				averageSpread = getAverage(spreadArray);
				stdDevSpread = getStdDev(spreadArray);
				
				NavigableMap<Double, Integer> sortedPriceHistoryMap = new TreeMap<Double, Integer>(priceHistoryMap);
				StringBuilder sb = new StringBuilder();
				
				int avgCount = (Collections.max(sortedPriceHistoryMap.values()) + Collections.min(sortedPriceHistoryMap.values()))/2;
				for (double j=sortedPriceHistoryMap.firstKey(); j<=sortedPriceHistoryMap.lastKey(); j=(double)Math.round((j+precision)*100)/100) {
					//System.out.println(j);
					//sb.append("["+f.format(j)+","+(f.format(np).equals(f.format(j))?avgCount:"null")+","+(sortedPriceHistoryMap.get(f.format(j))==null?"0":sortedPriceHistoryMap.get(f.format(j)))+"],");
					sb.append("["+f.format(j)+",null,"+(sortedPriceHistoryMap.get(j)==null?"0":sortedPriceHistoryMap.get(j))+"],");
				}
				tempString = sb.toString();
				tempString = tempString.substring(0, tempString.length()-1);
				//System.out.println(tempString);
				
				syncCache.put("spreadChartString_"+code, tempString,Expiration.byDeltaSeconds(86400));
				syncCache.put("spreadAvgCount_"+code, avgCount,Expiration.byDeltaSeconds(86400));
				syncCache.put("spreadPrecision_"+code, precision,Expiration.byDeltaSeconds(86400));
				
				syncCache.put("spreadAverage_"+code, averageSpread,Expiration.byDeltaSeconds(86400));
				syncCache.put("spreadStdDev_"+code, stdDevSpread,Expiration.byDeltaSeconds(86400));
				
				tempString = tempString.replaceAll("\\["+f.format(spread)+",null", "\\["+f.format(spread)+","+avgCount);
				return tempString;
			}
		}
		else{ // have cache
			if(np <= 1.0){
				f = new DecimalFormat("###.##");
			}
			else if(np <= 10.0){
				f = new DecimalFormat("###.##");
			}
			else if (np <= 200.0){
				f = new DecimalFormat("###.#");
			}
			else if (np <= 2000.0){
				f = new DecimalFormat("###");
			}
			tempString = cachedChartString;
			tempString = tempString.replaceAll("\\["+f.format(spread)+",null", "\\["+f.format(spread)+","+cachedAvgCount);
			return tempString;
		}
	}
	
	public String getTrendChartString(String code){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    String cachedChartString = (String) syncCache.get("trendChartString_"+code);
		
	    String tempString;

		// no cache, get from DB
		if (cachedChartString == null){
			Calendar cal = Calendar.getInstance();
			cal.add(Calendar.HOUR, +8);
			Date todayDate = cal.getTime();
			SimpleDateFormat sdfYear = new SimpleDateFormat("yyyy");
			SimpleDateFormat sdfMonth = new SimpleDateFormat("MM");
			SimpleDateFormat sdfDate = new SimpleDateFormat("dd");
			int toMonthInt = Integer.parseInt(sdfMonth.format(todayDate));
			toMonthInt--;
			String toYearString = sdfYear.format(todayDate);
			String toMonthString = String.format("%02d", toMonthInt);
			String toDateString = sdfDate.format(todayDate);
			
			cal.add(Calendar.YEAR, -1);
			Date lastYearDate = cal.getTime();
			
			int fromMonthInt = Integer.parseInt(sdfMonth.format(lastYearDate));
			fromMonthInt--;

			String fromYearString = sdfYear.format(lastYearDate);
			String fromMonthString = String.format("%02d", fromMonthInt);
			String fromDateString = sdfDate.format(lastYearDate); 
						
			
			List <PriceHistoryVO> priceHistoryList = getPriceHistoryVOList(code, fromYearString,fromMonthString,fromDateString,toYearString,toMonthString,toDateString,false);
			tempString = getTrendChartString(priceHistoryList);
			return tempString;
			
		}
		else{ // have cache
			tempString = cachedChartString;
			//tempString = tempString.replaceAll("\\["+f.format(spread)+",null", "\\["+f.format(spread)+","+cachedAvgCount);
			return tempString;
		}
	}	
	
	public String getTrendChartString(List<PriceHistoryVO> priceHistoryList){
	 	String code = priceHistoryList.get(0).getCode();
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		
	    String cachedChartString = (String) syncCache.get("trendChartString_"+code);
	    
	    String tempString;
	    
		// no cache, get from DB
		if (cachedChartString == null){
			
			if(priceHistoryList.isEmpty()){
				return "";
			}
			else{
				StringBuilder sb = new StringBuilder();
				for(int i=priceHistoryList.size()-1; i>=0; i--){
					sb.append("["+(priceHistoryList.size()-i)+","+priceHistoryList.get(i).getClosePrice()+"],");
				}
				tempString = sb.toString();
				tempString = tempString.substring(0, tempString.length()-1);
				syncCache.put("trendChartString_"+code, tempString,Expiration.byDeltaSeconds(86400));
				return tempString;
			}
		}
		else{ // have cache
			tempString = cachedChartString;
			return tempString;
		}
	}
	
	public Entry<Integer, Double> getTrendChartBestOrder(String code){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		int cachedOrder = (Integer) syncCache.get("trendChartOrder_"+code)==null?0:(Integer) syncCache.get("trendChartOrder_"+code);
	    double cachedR2 = (Double) syncCache.get("trendChartR2_"+code)==null?0.0:(Double) syncCache.get("trendChartR2_"+code);
		
	    Entry<Integer, Double> bestOrderEntry = null;

		// no cache, get from 
		if (cachedOrder == 0 || cachedR2 == 0.0){
			Calendar cal = Calendar.getInstance();
			cal.add(Calendar.HOUR, +8);
			Date todayDate = cal.getTime();
			SimpleDateFormat sdfYear = new SimpleDateFormat("yyyy");
			SimpleDateFormat sdfMonth = new SimpleDateFormat("MM");
			SimpleDateFormat sdfDate = new SimpleDateFormat("dd");
			int toMonthInt = Integer.parseInt(sdfMonth.format(todayDate));
			toMonthInt--;
			String toYearString = sdfYear.format(todayDate);
			String toMonthString = String.format("%02d", toMonthInt);
			String toDateString = sdfDate.format(todayDate);
			
			cal.add(Calendar.YEAR, -1);
			Date lastYearDate = cal.getTime();
			
			int fromMonthInt = Integer.parseInt(sdfMonth.format(lastYearDate));
			fromMonthInt--;

			String fromYearString = sdfYear.format(lastYearDate);
			String fromMonthString = String.format("%02d", fromMonthInt);
			String fromDateString = sdfDate.format(lastYearDate); 
						
			
			List <PriceHistoryVO> priceHistoryList = getPriceHistoryVOList(code, fromYearString,fromMonthString,fromDateString,toYearString,toMonthString,toDateString,false);
			bestOrderEntry = getTrendChartBestOrder(priceHistoryList);
			return bestOrderEntry;
			
		}
		else{ // have cache
			bestOrderEntry = new AbstractMap.SimpleEntry<Integer, Double>(cachedOrder,cachedR2);
			return bestOrderEntry;
		}
	}	
	
	public Entry<Integer, Double> getTrendChartBestOrder(List<PriceHistoryVO> priceHistoryList){
		String code = priceHistoryList.get(0).getCode();
	    Entry<Integer, Double> bestOrderEntry = null;
		if(priceHistoryList.isEmpty()){
			//bestOrderEntry.setValue(0,0.0); // if no price history, return 0
		}
		else{
			bestOrderEntry = getR2(priceHistoryList);
			//System.out.println(bestOrderEntry.getKey());
			//System.out.println(bestOrderEntry.getValue());
			syncCache.put("trendChartOrder_"+code, bestOrderEntry.getKey(),Expiration.byDeltaSeconds(86400));
			syncCache.put("trendChartR2_"+code, bestOrderEntry.getValue(),Expiration.byDeltaSeconds(86400));
		}
		return bestOrderEntry;
	}
	
	
	public long getProjectedVol(long cv){
		SimpleDateFormat sdf = new SimpleDateFormat("HH:mm");

		long morningMinutes = 150;
		long totalMinutes = 330;
		long openedMinutes = 0;
		
		try{
			Calendar cal = Calendar.getInstance();
			cal.add(Calendar.HOUR, +8);
			Date todayDate = cal.getTime();
			
			int dow = cal.get (Calendar.DAY_OF_WEEK);
			boolean isWeekday = ((dow >= Calendar.MONDAY) && (dow <= Calendar.FRIDAY));
			
			String currentTimeString = sdf.format(todayDate);
			
			Date morningStartTime = sdf.parse("09:30");
			Date afternoonStartTime = sdf.parse("13:00");
			Date currentTime = sdf.parse(currentTimeString);
			
			if(!isWeekday){
				openedMinutes = totalMinutes;
			}
			else if(Long.parseLong(currentTimeString.substring(0, 2))<9 || (Long.parseLong(currentTimeString.substring(0, 2))==9 && Long.parseLong(currentTimeString.substring(3, 5))<=30)){
				openedMinutes = totalMinutes;
			}
			else if(Long.parseLong(currentTimeString.substring(0, 2))>=9 && Long.parseLong(currentTimeString.substring(0, 2))<12){
				openedMinutes = (currentTime.getTime() - morningStartTime.getTime())/60000;
				if (openedMinutes <= 0){
					openedMinutes = 1;
				}
			}
			else if(Long.parseLong(currentTimeString.substring(0, 2))==12){
				openedMinutes = morningMinutes;
			}
			else if (Long.parseLong(currentTimeString.substring(0, 2))>=13 && Long.parseLong(currentTimeString.substring(0, 2))<16){
				openedMinutes = morningMinutes + (currentTime.getTime() - afternoonStartTime.getTime())/60000;
			}
			else if (Long.parseLong(currentTimeString.substring(0, 2))>=16){
				openedMinutes = totalMinutes;
			}
		}
		catch (Exception ex){
			ex.printStackTrace();
		}
		return (long) (cv * totalMinutes / openedMinutes);
	}
	
	public double getMiddlePrice(String code){
	    syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    double middlePrice;
	    try{
	    	middlePrice = (Double) syncCache.get("middlePrice_"+code);
	    }
	    catch (Exception ex){
	    	middlePrice = 0;
	    }
	    return middlePrice;
	}	
	
	public double getSpreadAverage(String code){
	    syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    double spreadAverage;
	    try{
	    	spreadAverage = (Double) syncCache.get("spreadAverage_"+code);
	    }
	    catch (Exception ex){
	    	spreadAverage = 0;
	    }
	    return spreadAverage;
	}
	
	public double getSpreadStdDev(String code){
	    syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    double spreadStdDev;
	    try{
	    	spreadStdDev = (Double) syncCache.get("spreadStdDev_"+code);
	    }
	    catch (Exception ex){
	    	spreadStdDev = 0;
	    }
	    return spreadStdDev;
	}
	
	public double getVolumeAverage(String code){
	    syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    double volumeAverage;
	    try{
	    	volumeAverage = (Double) syncCache.get("volumeAverage_"+code);
	    }
	    catch (Exception ex){
	    	volumeAverage = 0;
	    }
	    return volumeAverage;
	}
	
	public double getVolumeStdDev(String code){
	    syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    double volumeStdDev;
	    try{
	    	volumeStdDev = (Double) syncCache.get("volumeStdDev_"+code);	
	    }
	    catch (Exception ex){
	    	volumeStdDev = 0;
	    }
	    return volumeStdDev;
	}	
	
	public boolean hasHistory(String code){
	    syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));

	    String codeListString = (String) syncCache.get("codeListString");
	    String tempCodeList = "";
	    if (codeListString == null){ // no cache
	    	List<String> codeList = getCodeList();
	    	for (int i=0; i<codeList.size(); i++){
	    		tempCodeList += "," + codeList.get(i);
	    	}
	    	tempCodeList += ",";
	    	syncCache.put("codeListString", tempCodeList,Expiration.byDeltaSeconds(86400));
	    	
	    	if(tempCodeList.contains(","+code+",")){
	    		return true;
	    	}
	    	else{
	    		return false;
	    	}
	    }
	    else {
	    	if(codeListString.contains(","+code+",")){
	    		return true;
	    	}
	    	else{
	    		return false;
	    	}
	    }
		//priceHistoryDao = new PriceHistoryDao();
		//List<PriceHistoryVO> priceHistoryList = priceHistoryDao.getPriceHistoryList(code);
		//if(priceHistoryList.size()>0)
		//	return true;
		//else 
			
	}
	
	public void addPriceHistoryMass(String code, String fromYear, String fromMonth, String fromDate, String toYear, String toMonth, String toDate) {

		int codeInt = Integer.parseInt(code);
		URL url;
		HttpURLConnection connection = null;

		//List<String> dateList = new ArrayList<String>();
		//List<Double> closeList = new ArrayList<Double>();
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		//Date priceDate;

		try {
			// Create connection

			url = new URL("http://real-chart.finance.yahoo.com/table.csv?s="
					+ String.format("%04d", codeInt) + ".HK&a=" + fromMonth
					+ "&b=" + fromDate + "&c=" + fromYear + "&d=" + toMonth
					+ "&e=" + toDate + "&f=" + toYear + "&g=d&ignore=.csv");
						connection = (HttpURLConnection) url.openConnection();
			connection.setRequestMethod("GET");
			connection.setRequestProperty("Content-Type",
					"text/plain; charset=utf-8");
			connection.setRequestProperty("Content-Length",
					"" + Integer.toString("".getBytes().length));
			connection.setRequestProperty("Content-Language", "en-US");
			connection.setUseCaches(false);
			connection.setDoInput(true);
			connection.setDoOutput(true);

			// Send request
			DataOutputStream wr = new DataOutputStream(
					connection.getOutputStream());
			wr.writeBytes("");
			wr.flush();
			wr.close();

			// Get Response
			InputStream is = connection.getInputStream();
			BufferedReader rd = new BufferedReader(new InputStreamReader(is));
			String line;
			// StringBuffer response = new StringBuffer();
			String tempString[];
			while ((line = rd.readLine()) != null) {
				if (!line.startsWith("Date")) {
					tempString = line.split(",");
					if(!tempString[5].startsWith("0")){ // volume not equal to 0, means not public holiday
						//addPriceHistory(code, sdf.parse(tempString[0]), Double.parseDouble(tempString[4]), Long.parseLong(tempString[5]));
						addPriceHistory(code, sdf.parse(tempString[0]), Double.parseDouble(tempString[4]), Long.parseLong(tempString[5]), Double.parseDouble(tempString[3]), Double.parseDouble(tempString[2]));
					}
					//System.out.println(code+"\t"+ sdf.parse(tempString[0])+"\t"+ tempString[4]);
				}
			}
			rd.close();


		} catch (Exception e) {

			e.printStackTrace();

		} finally {

			if (connection != null) {
				connection.disconnect();
			}
			
			// remove the codeListString cache to let hasHistory to reload the list
			syncCache.delete("codeListString");
			
		}
	}
	
	public List<PriceHistoryVO> getPriceHistoryVOList(String code, String fromYear, String fromMonth, String fromDate, String toYear, String toMonth, String toDate, boolean fullList) {
		int codeInt = Integer.parseInt(code);
		URL url;
		HttpURLConnection connection = null; 
		List<PriceHistoryVO> priceHistoryVOList = new ArrayList<PriceHistoryVO>();
		PriceHistoryVO priceHistoryVO = null;

		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
	    String cachedYahooPriceHistory = (String) syncCache.get("yahooPriceHistory_"+code);
	    //System.out.println(cachedYahooPriceHistory);


	    if(cachedYahooPriceHistory == null){
		    try {
				System.out.println("Yahoo!! "+codeInt);
				url = new URL("http://real-chart.finance.yahoo.com/table.csv?s="
						+ String.format("%04d", codeInt) + ".HK&a=" + fromMonth
						+ "&b=" + fromDate + "&c=" + fromYear + "&d=" + toMonth
						+ "&e=" + toDate + "&f=" + toYear + "&g=d&ignore=.csv");
							connection = (HttpURLConnection) url.openConnection();
				connection.setRequestMethod("GET");
				connection.setRequestProperty("Content-Type","text/plain; charset=utf-8");
				connection.setRequestProperty("Content-Length", "" + Integer.toString("".getBytes().length));
				connection.setRequestProperty("Content-Language", "en-US");
				connection.setUseCaches(false);
				connection.setDoInput(true);
				connection.setDoOutput(true);
	
				// Send request
				DataOutputStream wr = new DataOutputStream(connection.getOutputStream());
				wr.writeBytes("");
				wr.flush();
				wr.close();
	
				// Get Response
				InputStream is = connection.getInputStream();
				BufferedReader rd = new BufferedReader(new InputStreamReader(is));
				String line;
				StringBuffer response = new StringBuffer();
				
				
				while ((line = rd.readLine()) != null) {
					if (!line.startsWith("Date")) {
						response.append(line);
						response.append("\n");
					}
				}
				rd.close();
				String yahooPriceHistory = new String();
				yahooPriceHistory = response.toString();
				//System.out.println(yahooPriceHistory);
				syncCache.put("yahooPriceHistory_"+code, yahooPriceHistory,Expiration.byDeltaSeconds(86400));
				cachedYahooPriceHistory = (String) syncCache.get("yahooPriceHistory_"+code);
		    } catch (Exception e) {
				//e.printStackTrace();
				// Do nothing when there is no information from yahoo finance history
			} finally {
				if (connection != null) {
					connection.disconnect();
				}
			}
	    }
	    
	    String tempYahooString[] = cachedYahooPriceHistory.split("\n");
	    //System.out.println(tempYahooString.length);
	    String tempString[];
	    try{
		    for(int i=0; i<tempYahooString.length; i++){		    
				tempString = tempYahooString[i].split(",");
				if(!tempString[5].startsWith("0")){ // volume not equal to 0, means not public holiday
					priceHistoryVO = new PriceHistoryVO();
					priceHistoryVO.setCode(code);
					priceHistoryVO.setClosePrice(Double.parseDouble(tempString[4]));
					priceHistoryVO.setLowPrice(Double.parseDouble(tempString[3]));
					priceHistoryVO.setHighPrice(Double.parseDouble(tempString[2]));
					priceHistoryVO.setVolume(Long.parseLong(tempString[5]));
					priceHistoryVO.setPriceDate(sdf.parse(tempString[0]));
					priceHistoryVOList.add(priceHistoryVO);
				}
			}
	    }
	    catch(Exception ex){
	    }
	    //System.out.println(priceHistoryVOList.size());
	    if(fullList)
	    	return priceHistoryVOList;
	    else
	    	return priceHistoryVOList.subList(0, priceHistoryVOList.size()<Integer.parseInt(System.getProperty("sampleSize"))?priceHistoryVOList.size():Integer.parseInt(System.getProperty("sampleSize")));
	}
	
	public List<String> getCodeList(){
		priceHistoryDao = new PriceHistoryDao();
		List<String> codeList = priceHistoryDao.getCodeList();
		return codeList;
	}
	
	/*
	public String getCodeFullList(int fromCode, int toCode){
		String fullCodeListString= (String)syncCache.get("fullCodeListString");
		if (fullCodeListString == null){
			fullCodeListString = "";
			//System.out.println("no cache");
			codeDao = new CodeDao();
			List<Long> codeFullList = codeDao.getCodeList(fromCode, toCode);
			for(int i=0; i<codeFullList.size(); i++){
				fullCodeListString += codeFullList.get(i)+",";
			}
			fullCodeListString = fullCodeListString.substring(0,fullCodeListString.length()-1);
			syncCache.put("fullCodeListString", fullCodeListString);
		}
		//System.out.println(fullCodeListString);
		return fullCodeListString;
	}
	*/
	
	public List<CodeVO> getCodeFullList(int fromCode, int toCode){
		String fullCodeListString= (String)syncCache.get("fullCodeListString_"+fromCode+"_"+toCode);
		List<CodeVO> codeList = new ArrayList<CodeVO>();
		if (fullCodeListString == null){
			fullCodeListString = "";
			codeDao = new CodeDao();
			codeList = codeDao.getCodeList(fromCode, toCode);
			for(int i=0; i<codeList.size(); i++){
				fullCodeListString += codeList.get(i).getCode()+",";
			}
			fullCodeListString = fullCodeListString.substring(0,fullCodeListString.length()-1);
			syncCache.put("fullCodeListString_"+fromCode+"_"+toCode, fullCodeListString);
		}
		else{
			String codeArray[] = fullCodeListString.split(",");
			for(int i=0; i<codeArray.length; i++){
				CodeVO codeVO = new CodeVO();
				codeVO.setCode(Long.parseLong(codeArray[i]));
				codeList.add(codeVO);
			}
		}
		return codeList;
	}
	
	// useless function
	public void deleteCodeHistory(String code){
		priceHistoryDao = new PriceHistoryDao();
		priceHistoryDao.deleteCodeHistory(code);
		
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		//System.out.println(code);
		//System.out.println(syncCache.get("chartString_"+code));
		syncCache.delete("chartString_"+code);
		//System.out.println(syncCache.get("chartString_"+code));
		syncCache.delete("volumeChartString_"+code);
		syncCache.delete("volumeChartString2_"+code);
		syncCache.delete("spreadChartString2_"+code);
		
		syncCache.delete("avgCount_"+code);
		syncCache.delete("precision_"+code);
		syncCache.delete("middlePrice_"+code);
		syncCache.delete("volumeAvgCount2_"+code);
		syncCache.delete("volumePrecision2_"+code);
		syncCache.delete("spreadAvgCount_"+code);
		syncCache.delete("spreadPrecision_"+code);		
		
		syncCache.delete("spreadAverage_"+code);
		syncCache.delete("spreadStdDev_"+code);
		syncCache.delete("volumeAverage_"+code);
		syncCache.delete("volumeStdDev_"+code);
		
		syncCache.delete("trendChartString_"+code);
		syncCache.delete("trendChartOrder_"+code);
		syncCache.delete("trendChartR2_"+code);
		
		syncCache.delete("yahooPriceHistory_"+code);
		
		syncCache.delete("codeListString");

	}
	
	public void deleteCodeFullList(){
		codeDao = new CodeDao();
		codeDao.deleteCodeList();
	}	
	
	public String refreshCodeFullList(){
		deleteCodeFullList();
		URL url;
		HttpURLConnection connection = null;
		codeDao = new CodeDao();
		String temp = "";

		try {
			url = new URL("https://www.hkex.com.hk/eng/market/sec_tradinfo/stockcode/eisdeqty_pf.htm");
			connection = (HttpURLConnection) url.openConnection();
			connection.setRequestMethod("GET");
			connection.setRequestProperty("Content-Length",	"" + Integer.toString("".getBytes().length));
			connection.setRequestProperty("Referer", "https://www.hkex.com.hk/eng/market/sec_tradinfo/stockcode/eisdeqty_pf.htm");
			connection.setUseCaches(false);
			connection.setReadTimeout(60000);
			connection.setConnectTimeout(60000);;

			// Send request
			BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream()));
			String line;
			String tempCode;
			CodeVO codeVO = new CodeVO();
			while ((line = rd.readLine()) != null) {
				//System.out.println(line);
				//temp += line;
				if (line.trim().startsWith("<td class=\"verd_black12\" width=\"18%\">")) {
					tempCode = line.trim().replaceAll("\\<.*?>","").replaceFirst("^0+(?!$)", "");
					if (!tempCode.equals("STOCK CODE")){
						//CodeVO codeVO = new CodeVO();
						codeVO.setCode(Long.parseLong(tempCode));
						codeDao.addCode(codeVO);
					}
				}
			}
			
			rd.close();
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (connection != null) {
				connection.disconnect();
			}
		}
		return temp;
	}	
	
	public double getAverage(double[] data){
		double sum = 0.0;
        for(double a : data)
            sum += a;
        return sum/data.length;
	}
	
	public double getVariance(double[] data){
        double mean = getAverage(data);
        double temp = 0;
        for(double a :data)
            temp += (mean-a)*(mean-a);
        return temp/data.length;
    }
	
	public double getStdDev(double[] data){
        return Math.sqrt(getVariance(data));
    }
	
	public double getStat(double value, double average, double sd){
		double stat = 0;
		/*
		if (value <= average+sd && value >= average-sd){
			stat = 0;
		}
		else if (value > average+sd && value <= average+(sd*2)){
			stat = 1;
		}
		else if (value > average+(sd*2)){
			stat = 2;
		}
		else if (value < average-sd && value >= average-(sd*2)){
			stat = -1;
		}
		else if (value < average-(sd*2)){
			stat = -2;
		}
		*/
		if (value <= average+sd/2 && value >= average-sd/2){
			stat = 0;
		}
		else if (value > average+sd/2 && value <= average+sd){
			stat = 0.5;
		}
		else if (value > average+sd && value <= average+(sd*1.5)){
			stat = 1;
		}
		else if (value > average+(sd*1.5) && value <= average+(sd*2)){
			stat = 1.5;
		}
		else if (value > average+(sd*2) && value <= average+(sd*2.5)){
			stat = 2;
		}
		else if (value > average+(sd*2.5) && value <= average+(sd*3)){
			stat = 2.5;
		}
		else if (value > average+(sd*3)){
			stat = 3;
		}		
		else if (value < average-(sd/2) && value >= average-sd){
			stat = -0.5;
		}
		else if (value < average-sd && value >= average-(sd*1.5)){
			stat = -1;
		}
		else if (value < average-(sd*1.5) && value >= average-(sd*2)){
			stat = -1.5;
		}
		else if (value < average-(sd*2) && value >= average-(sd*2.5)){
			stat = -2;
		}
		else if (value < average-(sd*2.5) && value >= average-(sd*3)){
			stat = -2.5;
		}	
		else if (value < average-(sd*3)){
			stat = -3;
		}	
	
		return stat;
	}
	
	public String getSuggestion(String code, double np, double spread, long cv, double diff, double previousClose, double dayHigh, double dayLow, double rsi10, double rsi20){
		String suggestion = "";
		
		double middlePrice = getMiddlePrice(code);

		double spreadAverage = getSpreadAverage(code);
		double spreadStdDev = getSpreadStdDev(code);
		
		double volumeAverage = getVolumeAverage(code);
		double volumeStdDev = getVolumeStdDev(code);
		
		double spreadStat = getStat(spread,spreadAverage,spreadStdDev);
		double volumeStat = getStat(cv,volumeAverage,volumeStdDev);
		
		if (middlePrice == 0 ||
			spreadAverage == 0 ||
			spreadStdDev == 0 ||
			volumeAverage == 0 ||
			volumeStdDev == 0){
			suggestion = "N"; // No cache
		}
		else {
			suggestion = "-";
			if(spreadStat <= -1.0 && volumeStat >= 2 && previousClose > dayHigh){
				// Selling climax: narrow spread, large volume, down bar, up trend
				suggestion = "B";
			}
			if(spreadStat <= -1.0 && volumeStat >= 2 && previousClose < dayLow){
				// Buying climax: narrow spread, large volume, up bar
				suggestion = "S";
			}
			if(spreadStat >= 1 && diff > 0){
				// Up-thrust: Wide spread and rise
				suggestion = "S";
			}			
		}
		//System.out.println(code+"\t"+spread+"\t"+spreadAverage+"\t"+spreadStdDev+"\t"+spreadStat+"\t"+cv+"\t"+volumeAverage+"\t"+volumeStdDev+"\t"+volumeStat+"\t"+suggestion);
		//System.out.println("Spread: "+code+" "+cv+" "+volumeAverage+" "+volumeStdDev+" "+volumeStat);
		
		//System.out.println("Suggestion: "+suggestion);
		return suggestion;
	}
	
	public void deleteCache(String cacheName){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		syncCache.delete(cacheName);
	}	
	
	public void addTask(List<CodeVO> codeList){
		Queue queue = QueueFactory.getDefaultQueue();
		for(int i=0; i<codeList.size(); i++){
			queue.add(withUrl("/task/refreshChartString.jsp").param("code", codeList.get(i).getCode()+""));
		}
	}
	
	
	
	public void addBuyList(String code, String name, String np, String volume){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		//String updatedBuyList = (String)syncCache.get("buyList")+"<button type=\"button\" onclick=\"addNewCode('"+code+"');\">+</button><span class=\"buylist\" onclick=\"togglePriceChart('"+code+"','"+np+"');\">"+code+" "+name+" "+np+" "+volume+"</span><br><span id=\"priceChart"+code+"\"></span>";
		String updatedBuyList = (String)syncCache.get("buyList")+"<button type=\"button\" onclick=\"addNewCode('"+code+"');\">+</button><span class=\"buylist\" >"+code+" "+name+" "+np+" "+volume+"</span><br><span id=\"priceChart"+code+"\"></span>";
		while(!syncCache.putIfUntouched("buyList", syncCache.getIdentifiable("buyList"),updatedBuyList)){
		}
	}	
	
	public String getBuyList(){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		if(syncCache.get("buyList") == null){
			return "No Cache<br/>";
		}
		else{
			String buyList = (String)syncCache.get("buyList");
			return buyList;
		}
	}	
	
	public void addSellList(String code, String name, String np, String volume){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		//String updatedSellList = (String)syncCache.get("sellList")+"<button type=\"button\" onclick=\"addNewCode('"+code+"');\">+</button><span class=\"selllist\" onclick=\"togglePriceChart('"+code+"','"+np+"');\">"+code+" "+name+" "+np+" "+volume+"</span><br><span id=\"priceChart"+code+"\"></span>";
		String updatedSellList = (String)syncCache.get("sellList")+"<button type=\"button\" onclick=\"addNewCode('"+code+"');\">+</button><span class=\"selllist\">"+code+" "+name+" "+np+" "+volume+"</span><br><span id=\"priceChart"+code+"\"></span>";
		while(!syncCache.putIfUntouched("sellList", syncCache.getIdentifiable("sellList"),updatedSellList)){
		}
	}	
	
	public String getSellList(){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		if(syncCache.get("sellList") == null){
			return "No Cache<br/>";
		}
		else{
			String sellList = (String)syncCache.get("sellList");
			return sellList;
		}
	}	
		
	
	public String getRunningStatus(){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		int remainingTask = QueueFactory.getDefaultQueue().fetchStatistics().getNumTasks();
		
		String runningStatus = "Not running";
		if(remainingTask > 0){
			runningStatus = " "+remainingTask +" left";
		}
		return runningStatus;
	}	
	
	 public Entry<Integer, Double> getR2(List<PriceHistoryVO> priceHistoryList) {

	        double[] x;// = { 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260 };
	        double[] y;// = { 79.15,79.25,79.6,79.8,79.35,79.1,80.1,80.1,80.75,81.7,82.25,83.2,83.6,83.5,82.75,82.05,83.55,83.3,82.4,81.95,82.2,81.85,82.6,82.85,83.9,83.5,83.75,82.8,82.45,83.2,83.7,83.4,82.85,83.2,83.65,83.6,83.2,83.65,83.3,83.6,82.95,82.95,82.25,82.65,82.6,82.5,82.35,82.9,82.8,84.2,83.55,82.85,82.3,82.2,82.15,80.7,79.55,79.55,79.55,78.65,79.1,79.05,79.15,79.8,78.4,78.2,78.6,78.6,77.65,77.9,77.55,77.75,78.4,78,77.95,78.3,78.85,79.05,78.45,78.85,78.65,78.25,77.9,78.1,77.95,78.1,78.05,78.5,78.2,77.85,77.55,77.2,77.05,76.85,76.5,77.1,77.05,77.95,77.55,77.05,76.1,76.4,76.55,77.15,77.05,77.35,76.6,76.45,75.8,75.15,74.1,72.6,72.55,72.15,73.4,74.1,74.25,74.25,74.25,74.25,74.95,74.1,74,74,74,72.9,71.75,71,71.2,71.1,71.1,71.3,70.85,70.6,70.2,69.95,70.2,70.6,71.9,73.1,72.6,73.3,73,72.3,72.45,71.25,71.5,72.4,72.2,72.85,72.8,71.85,70.25,70.8,71.75,71.7,72.05,71.95,71.95,71.95,72,69.45,70.05,69.9,69.8,69.55,69.75,69.8,67.9,67.8,66.6,67.1,65.95,65.55,65,64.65,65.05,65.9,66.6,66.45,67.35,67.1,67.35,67.35,67.35,66.95,66.65,66.55,67,67,67,67,68.25,68.55,69.2,70.2,70.65,70.8,70.4,70.55,70.45,70.85,70.45,70.8,73.8,76.45,76.4,77.45,76.95,76.95,75.6,74.85,74.45,74.75,76.55,75.75,75.15,74.75,74.8,75.65,75.15,74.65,74.6,74.25,75.15,75.15,74.6,74.15,73.45,74.15,73.85,73.55,73.9,73.75,73.1,73.55,73.25,72.85,73.15,73.25,72.6,71.95,71.9,72.4,72.5,73.45,73.7,73.65,73.05,71.95,70.25,70.15,70.15,69.55,69.55,67.9,68.45,65.75,67.75,68.1};
	        x = new double[priceHistoryList.size()];
	        y = new double[priceHistoryList.size()];
	        for(int i=priceHistoryList.size()-1; i>=0; i--){
	        	x[i] = priceHistoryList.size()-i;
	        	y[i] = priceHistoryList.get(i).getClosePrice();
	        	//System.out.println(x[i]+"\t"+y[i]);
	        	//System.out.println(x[i]+" "+y[i]);
	        }
	        
	        double r2 = 0.0;
	        Map<Integer, Double> priceHistoryMap = new HashMap<Integer, Double>();
	        RSquare regression;
	        Entry<Integer,Double> maxEntry = null;
	       
	        for (int i=1; i<=x.length; i++){
	        	try{
		        	regression = new RSquare(x, y, i);
		        	r2 = regression.R2();
		        	if(Double.isNaN(r2)){
		        		break;
		        	}
		        	//System.out.println(i+" "+r2);
		        	priceHistoryMap.put(i, r2);
		        	
	        	}
	        	catch (Exception ex){
	        	}
	        }
	        for(Entry<Integer,Double> entry : priceHistoryMap.entrySet()) {
	            if (maxEntry == null || entry.getValue() > maxEntry.getValue()) {
	                maxEntry = entry;
	            }
	        }
	        //System.out.println(maxEntry.getKey());
	        return maxEntry;
	    }
	
	public void sendMail(String subject, String body){
		Properties props = new Properties();
		Session session = Session.getDefaultInstance(props, null);
		String msgBody = body;
        try {
        	MimeMessage msg2 = new MimeMessage(session);
            msg2.setFrom(new InternetAddress("ccchungben@gmail.com"));
            msg2.addRecipient(Message.RecipientType.TO, new InternetAddress("admins"));
            msg2.setSubject(subject);
            msg2.setText(msgBody, "utf-8", "html");
            Transport.send(msg2);
        } catch (AddressException e) {
        	e.printStackTrace();
        } catch (MessagingException e) {
        	e.printStackTrace();
		}
	}
	
	public void addBoughtCode(String code, int numberOfStock, double buyPrice, Date buyDate) throws Exception{
        BoughtCodeDao boughtCodeDao = new BoughtCodeDao();
        boughtCodeDao.addBoughtCode(code, numberOfStock, buyPrice, buyDate);
        syncCache.delete("boughtCodeListString");
        
	}
	
	
	public String getBoughtCodeList() throws Exception{
		String boughtCodeListString = (String)syncCache.get("boughtCodeListString");
		if(boughtCodeListString == null){
			BoughtCodeDao boughtCodeDao = new BoughtCodeDao();
			List<BoughtCodeVO> boughtCodeList = boughtCodeDao.getBoughtCodeList();
			boughtCodeListString = new String();
			for(int i=0; i<boughtCodeList.size(); i++){
				//System.out.println(boughtCodeList.get(i).getCode()+" "+boughtCodeList.get(i).getBuyPrice());
				boughtCodeListString = boughtCodeListString+boughtCodeList.get(i).getCode()+",";
			}
			try{
				boughtCodeListString = boughtCodeListString.substring(0, boughtCodeListString.length()-1);
			}
			catch(Exception ex){
				boughtCodeListString = "";
			}
			syncCache.put("boughtCodeListString", boughtCodeListString);
		}
		//System.out.println(boughtCodeListString);
	   	return boughtCodeListString;
	}
	
	public String sellBoughtCode() throws Exception{
		String sellBoughtCode = new String();
		String sellList = getSellList();
		String boughtCodeList = getBoughtCodeList();  
		String boughtCodeArr[] = boughtCodeList.split(",");
		//System.out.println(sellList);
		//System.out.println(boughtCodeList);
		//System.out.println(boughtCodeArr.length);
		for(int i=0; i<boughtCodeArr.length; i++){
			//System.out.println(i+" "+boughtCodeArr[i]);
			if(sellList.contains("addNewCode('"+boughtCodeArr[i]+"')")){
				sellBoughtCode = sellBoughtCode + boughtCodeArr[i] + "<br/>";
				//System.out.println("Yeah");
			}
		}
		return sellBoughtCode;
	}
	
	public double getRSI(String code, int rsiTarget){
		syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
		if(syncCache.get("rsi"+rsiTarget+"_"+code) == null ){
			System.out.println("No RSI Cache " + rsiTarget);
		    // calculate for rsi
 			int rsiValue[] = {rsiTarget};
 			
			Calendar cal = Calendar.getInstance();
			cal.add(Calendar.HOUR, +8);
			Date todayDate = cal.getTime();
			SimpleDateFormat sdfYear = new SimpleDateFormat("yyyy");
			SimpleDateFormat sdfMonth = new SimpleDateFormat("MM");
			SimpleDateFormat sdfDate = new SimpleDateFormat("dd");
			
			int toMonthInt = Integer.parseInt(sdfMonth.format(todayDate));
			toMonthInt--;
			
			String toYearString = sdfYear.format(todayDate);
			String toMonthString = String.format("%02d", toMonthInt);
			String toDateString = sdfDate.format(todayDate);
			
			cal.add(Calendar.YEAR, -1);
			Date lastYearDate = cal.getTime();
			
			int fromMonthInt = Integer.parseInt(sdfMonth.format(lastYearDate));
			fromMonthInt--;

			String fromYearString = sdfYear.format(lastYearDate);
			String fromMonthString = String.format("%02d", fromMonthInt);
			String fromDateString = sdfDate.format(lastYearDate); 

			List <PriceHistoryVO> priceHistoryList = getPriceHistoryVOList(code, fromYearString,fromMonthString,fromDateString,toYearString,toMonthString,toDateString,true);
 			
			//priceHistoryList = priceHistoryList.subList(0, priceHistoryVOList.size()<Integer.parseInt(System.getProperty("sampleSize"))?priceHistoryVOList.size():Integer.parseInt(System.getProperty("sampleSize")));
			
 			double rise[] = new double[rsiTarget];
 			double drop[] = new double[rsiTarget];
 			for(int i=0; i<rsiTarget; i++){
 				rise[i]=0;
 				drop[i]=0;
 				if(priceHistoryList.get(i+1).getClosePrice()>=priceHistoryList.get(i+2).getClosePrice()){
 					rise[i] = priceHistoryList.get(i+1).getClosePrice()-priceHistoryList.get(i+2).getClosePrice();
					//System.out.println(i+" rise "+rise[i]);

 				}
 				else{
 					drop[i] = priceHistoryList.get(i+2).getClosePrice()-priceHistoryList.get(i+1).getClosePrice();
 					//System.out.println(i+" drop "+drop[i]);
 				}
 			}
 			
 			double rsi=0;
 			double totalRise=0;
 			double totalDrop=0;
 			for(int i=0; i<rsiValue.length; i++){
 				for(int j=0; j<rise.length; j++){
 					totalRise += rise[j];
 					totalDrop += drop[j];
 				}
 				rsi = (totalRise*100)/(totalRise+totalDrop);
 			}
 			syncCache.put("rsi"+rsiTarget+"_"+code, rsi);
 			System.out.println(syncCache.get("rsi"+rsiTarget+"_"+code));
 			return (Double) syncCache.get("rsi"+rsiTarget+"_"+code);

		}
		else{
			System.out.println("Have RSI Cache "+rsiTarget);
			System.out.println(syncCache.get("rsi"+rsiTarget+"_"+code));
			return (Double) syncCache.get("rsi"+rsiTarget+"_"+code);
		}
	}	
	
	
		
}
